/*Z OK。引导核和其它核启动入口 */

/*
 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
 *
 * SPDX-License-Identifier: GPL-2.0-only
 */

/* The kernel expects to be booted by a Multiboot compliant bootloader.
 * See Multiboot specifications:
 * www.gnu.org/software/grub/manual/multiboot
 * www.gnu.org/software/grub/manual/multiboot2
 *
 * The multiboot header's flags field is set to 3, indicating that we want
 * modules loaded on page boundaries, access to memory map information, and
 * information about the video mode table. Bit 16 of the multiboot header is
 * not set, indicating that the structure of the image should be taken from its
 * ELF headers.
 *
 * When the bootloader jumps to the entry point it is not in long mode and
 * 64-bit instructions are not accessible (Multiboot 1 does not have support
 * for this). While in protected mode, setup including the initialisation of
 * 64-bit paging structures is done before manually enabling long mode and
 * continuing. *//*Z 内核启动代码的第一部分，汇编代码，主要完成CPU特性、段页模式、虚拟空间布局、运行模式、栈等设置
 Multiboot兼容的boot loader创建的环境是(非EFI)：EAX-multiboot识别记号，EBX-引导信息块的物理地址，所有段寄存器-基地址0、限长4G，
 A20 gate使能，CR0禁用分页、使能保护模式，EFLAGS禁用Virtual8086模式、禁用中断。
 注意：ESP、GDTR、IDTR是未定义的 */
#include <config.h>
#include <machine/assembler.h>

#define IA32_EFER_MSR 0xC0000080
#define IA32_APIC_BASE_MSR 0x01B
#define APIC_ID_OFFSET 0x020

.section .phys.text

.code32
/*Z 通过端口0x3F8打印 */
BEGIN_FUNC(print_string)
    movw $0x3f8, %dx
1:
    movb (%ebx), %al
    outb %al, %dx
    incl %ebx
    decl %ecx
    jnz  1b
    ret
END_FUNC(print_string)
/*Z 本逻辑cpu死机 */
BEGIN_FUNC(hang)
1:
    hlt     /*Z 本逻辑cpu停机。有NMI等发生时会继续 */
    jmp  1b
END_FUNC(hang)
/*Z 检查CPU要支持1G巨页 */
#ifdef CONFIG_HUGE_PAGE
BEGIN_FUNC(huge_page_check)
    movl $0x80000001, %eax  /*Z cpuid扩展信息 */
    cpuid
    andl $0x04000000, %edx  /*Z 是否支持1G页 */
    jz   1f
    ret
1:
    movl $huge_page_error_string, %ebx
    movl $huge_page_error_size, %ecx
    call print_string
    call hang

huge_page_error_string:
    .string "Huge page not supported by the processor"
    .set huge_page_error_size, . - huge_page_error_string
END_FUNC(huge_page_check)
#endif /* CONFIG_HUGE_PAGE */
/*Z 设置2M页的四级分页模式，将物理地址的头4G全部映射到页表中 */
BEGIN_FUNC(setup_pml4)
#ifdef CONFIG_HUGE_PAGE
    call huge_page_check
#endif /* CONFIG_HUGE_PAGE */
    movl %cr0, %eax
    andl $0x7fffffff, %eax  /*Z 禁用分页模式，即所有地址视为物理地址 */
    movl %eax, %cr0

    movl $boot_pml4, %edi
    movl $0x0, %edx
    movl $1024, %ecx
1:
    movl %edx, (%edi)       /*Z 清零引导时用的一级页表 */
    addl $4, %edi
    loop 1b

    movl $boot_pdpt, %edi
    movl $1024, %ecx
1:
    movl %edx, (%edi)       /*Z 清零引导时用的二级页表 */
    addl $4, %edi
    loop 1b
/*Z 一级1页     二级1页        三级4页                   虚拟地址范围                大小            用途
 47     39     38     30      29     21
 000000000     000000000  --  000000000    00 00 00 00 00 00 -- 00 00 FF FF FF FF     4G        实际就是物理地址
                              111111111    00 7F 80 00 00 00 -- 00 7F BF FF FF FF  上面的头1G

               000000001  --  000000000      
                              111111111
 100000000 --                              80 00 00 00 00 00 -- 80 00 FF FF FF FF  上面的4G
               000000010  --  000000000    80 7F 80 00 00 00 -- 80 7F BF FF FF FF  上面的头1G
                              111111111
                                 
               000000011  --  000000000
                              111111111
                         /
 111111111     111111110                   FF 80 00 00 00 00 -- FF 80 FF FF FF FF  上面的4G     内核空间基地址。占顶端512G
                                           FF FF 80 00 00 00 -- FF FF BF FF FF FF  上面的头1G   内核代码基地址。占顶端2G
*/
    movl $boot_pml4, %edi
    movl $boot_pdpt, %ecx
    orl  $0x7, %ecx         /*Z 一级页表的第1个表项指向二级页表，并设置允许用户访问和写模式 */
    movl %ecx, (%edi)
    movl %ecx, 0x800(%edi)  /*Z 并在第256、511项处做备份 */
    movl %ecx, 4088(%edi)

    movl $_boot_pd, %ecx
    orl  $0x7, %ecx
    movl $boot_pdpt, %edi
    movl %ecx, (%edi)       /*Z 二级页表的第1个表项指向三级页表，并设置允许用户访问和写模式 */
    movl %ecx, 4080(%edi)   /*Z 并在第510项处做备份 */
    addl $0x1000, %ecx
    movl %ecx, 8(%edi)
    addl $0x1000, %ecx
    movl %ecx, 16(%edi)
    addl $0x1000, %ecx
    movl %ecx, 24(%edi)      /*Z 二级页表的第2、3、4项指向三级页表（依次紧临上一页）*/

    /* Map first 4GiB into the _boot_pd. */
    movl $_boot_pd, %edi
    movl $2048, %ecx
    movl $0x87, %edx        /*Z 设置三级页表项均指向2M页幁（因此无四级页表）*/
2:
    movl %edx, (%edi)
    addl $0x200000, %edx    /*Z 将物理地址的头4G放入三级页表中 */
    addl $8, %edi
    loop 2b
    ret                     /*Z 至此，内核可转入分页模式 */
END_FUNC(setup_pml4)
/*Z 检查CPU要支持PCID，即CPU在CR3寄存器低12位保存的用于区别不同进程上下文的id */
BEGIN_FUNC(pcid_check)
    movl $0x1, %eax
    xorl %ecx, %ecx
    cpuid
    andl $0x20000, %ecx
    jz   1f
    ret
1:
    movl $pcid_error_string, %ebx
    movl $pcid_error_size, %ecx
    call print_string
    call hang

pcid_error_string:
    .string "PCIDs not supported by the processor"
    .set pcid_error_size, . - pcid_error_string
END_FUNC(pcid_check)
/*Z 检查CPU要支持INVPCID指令（使无效TLB和页表缓存中相应PCID进程的条目） */
BEGIN_FUNC(invpcid_check)
    movl $0x7, %eax
    xorl %ecx, %ecx
    cpuid
    andl $0x400, %ebx
    jz   1f
    ret
1:
    movl  $invpcid_error_string, %ebx
    movl  $invpcid_error_size, %ecx
    call  print_string
    call  hang

invpcid_error_string:
    .string "INVPCID instruction not supported by the processor"
    .set invpcid_error_size, . - invpcid_error_string
END_FUNC(invpcid_check)
/*Z 检查CPU要支持SYSCALL/SYSRET指令，只有64位模式时有 */
BEGIN_FUNC(syscall_check)
    movl $0x80000001, %eax
    xorl %ecx, %ecx
    cpuid
    andl $0x20000000, %edx  /*Z 64位架构 */
    jz   1f
    ret
1:
    movl  $syscall_error_string, %ebx
    movl  $syscall_error_size, %ecx
    call  print_string
    call  hang

syscall_error_string:
    .string "SYSCALL/SYSRET instruction not supported by the processor"
    .set syscall_error_size, . - syscall_error_string
END_FUNC(syscall_check)
/*Z 检查CPU要支持RDFSBASE/RDGSBASE/WRFSBASE/WRGSBASE指令，即读写fs, gs段寄存器，仅64位模式下 */
#ifdef CONFIG_FSGSBASE_INST
BEGIN_FUNC(fsgsbase_enable)
    movl $0x7, %eax     /*Z 获取CPU扩展特性 */
    xorl %ecx, %ecx
    cpuid
    andl $1, %ebx
    jz   1f             /*Z 如CPU不支持读写fs, gs段寄存器，则报错死机 */
    movl %cr4, %eax
    /* Enable the bit in cr4. */
    orl  $0x10000, %eax
    movl %eax, %cr4     /*Z 使能读写fs, gs段寄存器的专用指令 */
    ret
1:
    movl $fsgsbase_error_string, %ebx
    movl $fsgsbase_error_size, %ecx
    call print_string
    call hang

fsgsbase_error_string:
    .string "fsgsbase instructions not supported by the processor"
    .set fsgsbase_error_size, . - fsgsbase_error_string
END_FUNC(fsgsbase_enable)
#endif /* CONFIG_FSGSBASE_INST */
/*Z 使能SYSCALL/SYSRET指令 */
BEGIN_FUNC(syscall_enable)
    call syscall_check
    /* Set SCE (bit 0) in the extended feature MSR. */
    movl $IA32_EFER_MSR, %ecx   /*Z 0xC0000080 */
    rdmsr
    orl $0x1, %eax
    wrmsr
    ret
END_FUNC(syscall_enable)
/*Z 设置、使能64位模式 */
BEGIN_FUNC(enable_x64_mode)
#ifdef CONFIG_SUPPORT_PCID
    call pcid_check
    call invpcid_check
#endif
    /* Put base pointer in cr3. */
    movl $boot_pml4, %eax       /*Z 在CR3中设置引导时用的一级页表 */
    movl %eax, %cr3
    /* Set PAE (bit 5), as this is required before switching to long mode. */
    movl %cr4, %eax
    orl $0x20, %eax             /*Z 使能超过32位的物理地址 */
    movl %eax, %cr4
    /* Set LME (bit 8) in the extended feature MSR. */
    movl $IA32_EFER_MSR, %ecx   /*Z 使能IA-32e模式 */
    rdmsr
    orl $0x100, %eax
    wrmsr
    /* Set PG (bit 31) of cr0 to enable paging. */
    movl %cr0, %eax
    orl $0x80000000, %eax       /*Z 使能分页 */
    movl %eax, %cr0
#ifdef CONFIG_SUPPORT_PCID
    /* Enable PCID (bit 17), must be done in long mode. */
    movl %cr4, %eax
    orl  $0x20000, %eax         /*Z 使能PCID */
    movl %eax, %cr4
#endif
    ret
END_FUNC(enable_x64_mode)
/*Z fsgs段寄存器、分页、64位、GDT、SYSCALL等初始化 */
BEGIN_FUNC(common_init)
    /* Disable paging. */       /*Z 禁用分页，CR0的第31位置0 */
    movl %cr0, %eax
    andl $0x7fffffff, %eax
    movl %eax, %cr0

#ifdef CONFIG_FSGSBASE_INST
    call fsgsbase_enable        /*Z 使能fs, gs寄存器读写指令 */
#endif /* CONFIG_FSGSBASE_INST */

    /* Initialize boot PML4 and switch to long mode. */
    call setup_pml4             /*Z 设置页表 */
    call enable_x64_mode        /*Z 启用64位模式，使能分页 */
    lgdt _gdt64_ptr             /*Z 加载全局描述符表寄存器GDTR */

#ifdef CONFIG_SYSCALL
    call syscall_enable         /*Z 使能SYSCALL指令 */
#endif

    ret
END_FUNC(common_init)
/*Z x86_64内核入口函数 */
BEGIN_FUNC(_start)
    /* Assume we are MultiBooted, e.g. by GRUB.
     * While not immediately checked, the magic number is checked prior to 
     * Multiboot dependent operations. */
    movl %eax, %edi /* multiboot_magic    */
    movl %ebx, %esi /* multiboot_info_ptr */

    /* Load kernel boot stack pointer. *//*Z 设置栈 */
    leal boot_stack_top, %esp

    /* Reset EFLAGS register (also disables interrupts etc.). *//*Z 设置EFLAGS寄存器 */
    pushl $0
    popf

    /* Already push parameters for calling boot_sys later. Push
     * them as 8 byte values so we can easily pop later. *//*Z 将grub准备的有关信息传递给boot_sys */
    pushl $0
    pushl %esi /* 2nd parameter: multiboot_info_ptr */
    pushl $0
    pushl %edi /* 1st parameter: multiboot_magic    */

    call common_init

    /* Reload CS with long bit to enable long mode. */
    ljmp $8, $_start64          /*Z 跳转到下一步启动代码，并设置CS段寄存器指向GDT第二项，从而完成64位模式下的段寻址模式设置 */
END_FUNC(_start)
/*Z 64位启动代码，主要功能是从低端地址跳转到高端地址 */
.code64
.align 4096
BEGIN_FUNC(_start64)
    /* Leave phys code behind and jump to the high kernel virtual address. */
    movabs $_entry_64, %rax
    jmp *%rax
END_FUNC(_start64)

.section .phys.data
_gdt64_ptr:                     /*Z 用于加载全局段描述符表寄存器GDTR的操作数 */
    .word (3 * 8) - 1           /*Z 低两个字节为limit，指明GDT长度24字节 */
    .long _gdt64                /*Z 高八个字节为base(这里只用了4个字节，不好)，指明GDT的基地址 */

.align 16
_gdt64:                         /*Z 全局段描述符表GDT */
    .quad 0x0000000000000000    /*Z 第一项置0未用 */
    .word   0
    .word   0
    .byte   0
    .byte   0x98
    .byte   0x20
    .byte   0                   /*Z 第二项描述一个纯64位特权模式的只可执行的代码段，段基地址0 */
    .word   0
    .word   0
    .byte   0
    .byte   0x90
    .byte   0
    .byte   0                   /*Z 第三项描述一个特权模式的只读数据段，段基地址0 */

.section .phys.bss
.align 4096
_boot_pd:
    .fill 16384         /*Z 引导时用的三级页表，4页，清零 */

.section .boot.text /*Z 这节代码在虚拟地址空间 */
/*Z 64位启动代码，从这开始使用的都是虚拟空间顶端2G的地址，见linker.lds；物理地址在1M之上， */
BEGIN_FUNC(_entry_64)
    /* Update our stack pointer. */
    movq $0xffffffff80000000, %rax      /*Z 内核代码基地址，位于虚拟地址空间的顶端2G */
    addq %rax, %rsp                     /*Z + 物理地址 = 虚拟地址，内核栈、幁映射入内核虚拟空间。前面已建立页表 */
    addq %rax, %rbp

    /* Pop the multiboot parameters off. *//*Z 这是遵从有关ABI通过rdi, rsi等寄存器传参 */
    pop %rdi
    pop %rsi

    /* Load our real kernel stack. */
    leaq kernel_stack_alloc + (1 << CONFIG_KERNEL_STACK_BITS), %rsp /*Z 调整rsp指向实际栈栈底 */

    movabs $restore_user_context, %rax       /*Z 将restore_user_context()作为返回地址入栈。这里的jmp模拟的就是函数调用 */
    push %rax
    jmp boot_sys                            /*Z 转入C启动代码 */
END_FUNC(_entry_64)

.section .phys.text

#ifdef ENABLE_SMP_SUPPORT
/*Z 其它核启动代码，在启动的最后阶段由引导核激活 */
BEGIN_FUNC(boot_cpu_start)/*Z 编译时在地址在1M以上，运行时搬移到0x1000，实模式，CS:IP=0x0100:0x0000 */
.code16 /*Z 加载GDTR */
    /* Set DS equal to CS and load GDTR register with GDT pointer. */
    movw %cs, %ax
    movw %ax, %ds
    lgdt _boot_gdt_ptr - boot_cpu_start
    /*Z 使能保护模式 */
    /* Enable protected mode. */
    movl %cr0, %eax
    orl  $1,   %eax
    movl %eax, %cr0
    /*Z 长跳转使CPU通过GDT重新加载CS */
    /* Reload CS with a far jump. */
    ljmpl $0x08, $1f    /*Z 段选择子8表示第1个GDT项，特权 */

.code32
1:  /*Z 设置DS/ES/SS段 */
    /* Load DS/ES/SS with kernel data segment selector. */
    movw $0x10, %ax
    movw %ax,   %ds
    movw %ax,   %es
    movw %ax,   %ss
    /*Z 设置栈 */
    /* Use temporary kernel boot stack pointer. */
    leal boot_stack_top, %esp
    /*Z 重置标志寄存器，屏蔽中断 */
    /* Reset EFLAGS register (also disables interrupts etc.). */
    pushl $0
    popf
    /*Z fsgs段寄存器、分页、64位、GDT、SYSCALL等初始化 */
    call common_init
    /*Z 长跳转使CPU通过统一的GDT重新加载CS，完成64位模式设置 */
    /* Reload CS with long bit to enable long mode. */
    ljmp $8, $_start_ap64
    jmp 1b  /*Z 不好：没用的指令 */
END_FUNC(boot_cpu_start)

.code64 /*Z 这是为了跳入内核虚拟地址空间 */
BEGIN_FUNC(_start_ap64)
    /* Leave phys code behind and jump to the high kernel virtual address. */
    movabs $_entry_ap64, %rax
    jmp *%rax
END_FUNC(_start_ap64)

_boot_gdt_ptr:/*Z 其它核用于加载全局段描述符表寄存器GDTR的操作数 */
    .word   (3 * 8) - 1 /* Limit: 3 segments * 8 bytes - 1 byte *//*Z 低两个字节为limit，指明GDT长度24字节 */
    .long   _boot_gdt   /* Address of boot GDT *//*Z 高4个字节为base，指明GDT的基地址。不好：这里的链接地址在1M以上，
    此时已作为可再利用区加入了untyped列表，虽然此时无人typed。符号地址都有同样问题????? */
/* GDT for getting us through 32-bit protected mode. */
    .align 16
_boot_gdt:
    .quad 0x0000000000000000 /* Null segment *//*Z 第一项置0未用 */
    .quad 0x00cf9b000000ffff /* 4GB kernel code segment *//*Z 第二项描述一个32位特权模式的只可执行的代码段，段基地址0长4G */
    .quad 0x00cf93000000ffff /* 4GB kernel data segment *//*Z 第三项同上的读写数据段 */

.global boot_cpu_end
boot_cpu_end:

.section .boot.text /*Z 这节代码在虚拟地址空间 */

BEGIN_FUNC(_entry_ap64)
    /* Get the index of this cpu. *//*Z 获取当前cpu索引，因为内核锁只有一个cpu在执行 */
    movq smp_aps_index, %rcx
    /*Z 设置内核栈 */
    /* Switch to a real kernel stack. */
    leaq kernel_stack_alloc, %rsp
    inc %rcx
    shlq $CONFIG_KERNEL_STACK_BITS, %rcx
    addq %rcx, %rsp
    /*Z 将restore_user_context()作为返回地址入栈。这里的jmp模拟的就是函数调用 */
    movabs $restore_user_context, %rax
    push %rax
    jmp boot_node   /*Z 跳到smp_sys.c的C代码 */
END_FUNC(_entry_ap64)

#endif /* ENABLE_SMP_SUPPORT */
